{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"temp\" + os.sep\n",
    "\n",
    "TREE_MODEL_FILE = \"tree_model_19.ak\"\n",
    "PIPELINE_MODEL_FILE = \"pipeline_model_19.ak\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_5_1\n",
    "# https://www.yuque.com/pinshu/alink_tutorial/book_python_2_5_1\n",
    "\n",
    "source = CsvSourceBatchOp()\\\n",
    "    .setFilePath(\"http://archive.ics.uci.edu/ml/machine-learning-databases\"\n",
    "        + \"/iris/iris.data\")\\\n",
    "    .setSchemaStr(\"sepal_length double, sepal_width double, petal_length double, \"\n",
    "        + \"petal_width double, category string\")\n",
    "\n",
    "source\\\n",
    "    .lazyPrint()\\\n",
    "    .lazyPrint(title=\">>> print with title.\")\\\n",
    "    .lazyPrint(2)\\\n",
    "    .lazyPrint(2, \">>> print 2 rows with title.\")\\\n",
    "    .lazyPrintStatistics()\\\n",
    "    .lazyPrintStatistics(\">>> summary of current data.\")\\\n",
    "    .lazyCollectToDataframe(lambda df : print(\"number of rows : \" + str(len(df))))\\\n",
    "    .lazyCollectStatistics(lambda tableSummary : \n",
    "                           print(\"number of valid values :\" \n",
    "                                 + str(tableSummary.numValidValue(\"sepal_length\")) \n",
    "                                 + \"\\nnumber of missing values :\" \n",
    "                                 + str(tableSummary.numMissingValue(\"sepal_length\"))))\\\n",
    "    .link(\n",
    "        SelectBatchOp()\\\n",
    "            .setClause(\"sepal_length, sepal_width, sepal_length/sepal_width AS ratio\")\n",
    "    )\\\n",
    "    .lazyPrint(title=\">>> final data\")\\\n",
    "    .lazyPrintStatistics(\">>> summary of final data.\")\n",
    "\n",
    "BatchOperator.execute()\n",
    "\n",
    "Pipeline()\\\n",
    "    .add(\n",
    "        Select()\\\n",
    "            .setClause(\"sepal_length, sepal_width, sepal_length/sepal_width AS ratio\")\\\n",
    "            .enableLazyPrintTransformData(5, \">>> output data after Select\")\\\n",
    "            .enableLazyPrintTransformStat(\">>> summary of data after Select \")\n",
    "    )\\\n",
    "    .add(\n",
    "        StandardScaler()\\\n",
    "            .setSelectedCols([\"sepal_length\", \"sepal_width\"])\\\n",
    "            .enableLazyPrintModelInfo(\">>> model info\")\\\n",
    "            .enableLazyPrintTransformData(5, \">>> output data after StandardScaler\")\\\n",
    "            .enableLazyPrintTransformStat(\">>> summary of data after StandardScaler\")\\\n",
    "    )\\\n",
    "    .fit(source)\\\n",
    "    .transform(source)\\\n",
    "    .lazyPrint(title=\">>> output data after the whole pipeline\")\n",
    "\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_6\n",
    "\n",
    "df = pd.DataFrame(\n",
    "    [\n",
    "        [\"sunny\", 85.0, 85.0, False, \"no\"],\n",
    "        [\"sunny\", 80.0, 90.0, True, \"no\"],\n",
    "        [\"overcast\", 83.0, 78.0, False, \"yes\"],\n",
    "        [\"rainy\", 70.0, 96.0, False, \"yes\"],\n",
    "        [\"rainy\", 68.0, 80.0, False, \"yes\"],\n",
    "        [\"rainy\", 65.0, 70.0, True, \"no\"],\n",
    "        [\"overcast\", 64.0, 65.0, True, \"yes\"],\n",
    "        [\"sunny\", 72.0, 95.0, False, \"no\"],\n",
    "        [\"sunny\", 69.0, 70.0, False, \"yes\"],\n",
    "        [\"rainy\", 75.0, 80.0, False, \"yes\"],\n",
    "        [\"sunny\", 75.0, 70.0, True, \"yes\"],\n",
    "        [\"overcast\", 72.0, 90.0, True, \"yes\"],\n",
    "        [\"overcast\", 81.0, 75.0, False, \"yes\"],\n",
    "        [\"rainy\", 71.0, 80.0, True, \"no\"]\n",
    "    ]\n",
    ")\n",
    "\n",
    "source = BatchOperator.fromDataframe(df, schemaStr=\"outlook string, Temperature double, Humidity double, Windy boolean, play string\")\n",
    "\n",
    "source\\\n",
    "    .link(\n",
    "        C45TrainBatchOp()\\\n",
    "            .setFeatureCols([\"outlook\", \"Temperature\", \"Humidity\", \"Windy\"])\\\n",
    "            .setCategoricalCols([\"outlook\", \"Windy\"])\\\n",
    "            .setLabelCol(\"play\")\n",
    "    )\\\n",
    "    .link(\n",
    "        AkSinkBatchOp()\\\n",
    "            .setFilePath(DATA_DIR + TREE_MODEL_FILE)\\\n",
    "            .setOverwriteSink(True)\n",
    "    )\n",
    "BatchOperator.execute()\n",
    "\n",
    "AkSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + TREE_MODEL_FILE)\\\n",
    "    .link(\n",
    "        DecisionTreeModelInfoBatchOp()\\\n",
    "            .lazyPrintModelInfo()\\\n",
    "            .lazyCollectModelInfo(\n",
    "                lambda decisionTreeModelInfo: \n",
    "                    decisionTreeModelInfo.saveTreeAsImage(\n",
    "                        DATA_DIR + \"tree_model.png\", True)\n",
    "            )\n",
    "    )\n",
    "BatchOperator.execute()\n",
    "\n",
    "\n",
    "if os.path.exists(DATA_DIR + PIPELINE_MODEL_FILE):\n",
    "    os.remove(DATA_DIR + PIPELINE_MODEL_FILE)\n",
    "\n",
    "\n",
    "df = pd.DataFrame(\n",
    "    [\n",
    "        [2009, 0.5],\n",
    "        [2010, 9.36],\n",
    "        [2011, 52.0],\n",
    "        [2012, 191.0],\n",
    "        [2013, 350.0],\n",
    "        [2014, 571.0],\n",
    "        [2015, 912.0],\n",
    "        [2016, 1207.0],\n",
    "        [2017, 1682.0]\n",
    "    ]\n",
    ")  \n",
    "train_set = BatchOperator.fromDataframe(df, schemaStr='x int, gmv double')\n",
    "\n",
    "pipeline = Pipeline()\\\n",
    "    .add(\n",
    "        Select().setClause(\"*, x*x AS x2\")\n",
    "    )\\\n",
    "    .add(\n",
    "        LinearRegression()\\\n",
    "            .setFeatureCols([\"x\", \"x2\"])\\\n",
    "            .setLabelCol(\"gmv\")\\\n",
    "            .setPredictionCol(\"pred\")\n",
    "    )\n",
    "\n",
    "pipeline.fit(train_set).save(DATA_DIR + PIPELINE_MODEL_FILE)\n",
    "BatchOperator.execute()\n",
    "\n",
    "pipelineModel = PipelineModel.load(DATA_DIR + PIPELINE_MODEL_FILE);\n",
    "\n",
    "stages = pipelineModel.getTransformers()\n",
    "\n",
    "for i in range(2) :\n",
    "    print(str(i) + \"\\t\" + str(stages[i]));\n",
    "\n",
    "stages[1].getModelData()\\\n",
    "    .link(\n",
    "        LinearRegModelInfoBatchOp().lazyPrintModelInfo()\n",
    "    )\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
